# -*- coding: utf-8 -*-
"""
@author: haoyanghuang
@brief: utils for nlp

"""
import os
import re
import csv
import codecs
import numpy as np
import pandas as pd
from .dist_utils import _calc_similarity

def _get_embedd_Index(glove_dir):
    embedding_Index = {}
    with codecs.open(os.path.join(glove_dir), encoding='utf-8') as f:
        for line in f:
            values = line.split()
            try:
                word = values[0]
                coefs = np.asarray(values[1:], dtype='float32')
            except:
                continue
            embedding_Index[word] = coefs
        f.close()
    return embedding_Index


def _tokenize(text, token_pattern=" "):
    # token_pattern = r"(?u)\b\w\w+\b"
    # token_pattern = r"\w{1,}"
    # token_pattern = r"\w+"
    # token_pattern = r"[\w']+"
    if token_pattern == " ":
        # just split the text into tokens
        return text.split(" ")
    else:
        token_pattern = re.compile(token_pattern, flags = re.UNICODE | re.LOCALE)
        group = token_pattern.findall(text)
        return group

# after spacy not become !

